463d52
@@ -30,11 +30,12 @@
import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -86,6 +87,8 @@
public class ReplicationSourceManager implements ReplicationListener {
   private final Stoppable stopper;
   // All logs we are currently tracking
   private final Map<String, SortedSet<String>> hlogsById;
+  // Logs for recovered sources we are currently tracking
+  private final Map<String, SortedSet<String>> hlogsByIdRecoveredQueues;
   private final Configuration conf;
   private final FileSystem fs;
   // The path to the latest log we saw, for new coming sources
@@ -126,6 +129,7 @@
public class ReplicationSourceManager implements ReplicationListener {
     this.replicationTracker = replicationTracker;
     this.stopper = stopper;
     this.hlogsById = new HashMap<String, SortedSet<String>>();
+    this.hlogsByIdRecoveredQueues = new ConcurrentHashMap<String, SortedSet<String>>();
     this.oldsources = new CopyOnWriteArrayList<ReplicationSourceInterface>();
     this.conf = conf;
     this.fs = fs;
@@ -177,20 +181,29 @@
public class ReplicationSourceManager implements ReplicationListener {
    * @param id id of the peer cluster
    * @param queueRecovered Whether this is a recovered queue
    */
-  public void cleanOldLogs(String key,
-                           String id,
-                           boolean queueRecovered) {
-    synchronized (this.hlogsById) {
-      SortedSet<String> hlogs = this.hlogsById.get(id);
-      if (queueRecovered || hlogs.first().equals(key)) {
-        return;
+  public void cleanOldLogs(String key, String id, boolean queueRecovered) {
+    if (queueRecovered) {
+      SortedSet<String> hlogs = hlogsByIdRecoveredQueues.get(id);
+      if (hlogs != null && !hlogs.first().equals(key)) {
+        cleanOldLogs(hlogs, key, id);
       }
-      SortedSet<String> hlogSet = hlogs.headSet(key);
-      for (String hlog : hlogSet) {
-        this.replicationQueues.removeLog(id, hlog);
+    } else {
+      synchronized (this.hlogsById) {
+        SortedSet<String> hlogs = hlogsById.get(id);
+        if (!hlogs.first().equals(key)) {
+          cleanOldLogs(hlogs, key, id);
+        }
       }
-      hlogSet.clear();
     }
+ }
+  
+  private void cleanOldLogs(SortedSet<String> hlogs, String key, String id) {
+    SortedSet<String> hlogSet = hlogs.headSet(key);
+    LOG.debug("Removing " + hlogSet.size() + " logs in the list: " + hlogSet);
+    for (String hlog : hlogSet) {
+      this.replicationQueues.removeLog(id, hlog);
+    }
+    hlogSet.clear();
   }
 
   /**
@@ -285,6 +298,14 @@
public class ReplicationSourceManager implements ReplicationListener {
   protected Map<String, SortedSet<String>> getHLogs() {
     return Collections.unmodifiableMap(hlogsById);
   }
+  
+  /**
+   * Get a copy of the hlogs of the recovered sources on this rs
+   * @return a sorted set of hlog names
+   */
+  protected Map<String, SortedSet<String>> getHlogsByIdRecoveredQueues() {
+    return Collections.unmodifiableMap(hlogsByIdRecoveredQueues);
+  }
 
   /**
    * Get a list of all the normal sources of this rs
@@ -303,7 +324,6 @@
public class ReplicationSourceManager implements ReplicationListener {
   }
 
   void preLogRoll(Path newLog) throws IOException {
-
     synchronized (this.hlogsById) {
       String name = newLog.getName();
       for (ReplicationSourceInterface source : this.sources) {
@@ -416,6 +436,7 @@
public class ReplicationSourceManager implements ReplicationListener {
     LOG.info("Done with the recovered queue " + src.getPeerClusterZnode());
     this.oldsources.remove(src);
     deleteSource(src.getPeerClusterZnode(), false);
+    this.hlogsByIdRecoveredQueues.remove(src.getPeerClusterZnode());
   }
 
   /**
@@ -563,10 +584,12 @@
public class ReplicationSourceManager implements ReplicationListener {
             break;
           }
           oldsources.add(src);
-          for (String hlog : entry.getValue()) {
+          SortedSet<String> hlogsSet = entry.getValue();
+          for (String hlog : hlogsSet) {
             src.enqueueLog(new Path(oldLogDir, hlog));
           }
           src.startup();
+          hlogsByIdRecoveredQueues.put(peerId, hlogsSet);
         } catch (IOException e) {
           // TODO manage it
           LOG.error("Failed creating a source", e);
